#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: import_increment_company_list.py
@time: 2018/1/15 11:33
"""

import sys

import click

sys.path.append('..')
sys.path.append('../..')
from common import util

from common.mongo import MongDb
from logger import Logger

MONGO_DB_SOURCE = {
    "host": "172.16.215.2",
    "port": 40042,
    "db": "company_data",
    "username": "work",
    "password": "haizhi",
}

# 日志模块
log = Logger('import_increment_company_list.log').get_logger()

source_db = MongDb(MONGO_DB_SOURCE['host'], MONGO_DB_SOURCE['port'], MONGO_DB_SOURCE['db'],
                   MONGO_DB_SOURCE['username'],
                   MONGO_DB_SOURCE['password'], log=log)


@click.command()
@click.option('--filename',
              default='/home/youfeng/files/all_repeat_list.txt',
              help='企业名单文件路径')
@click.option('--startline',
              default=3144000,
              type=int,
              help='企业名单文件路径')
def main(filename, startline):
    # 3144000
    result_list = []
    count = 0
    total = 0
    source_table = 'new_gsxt_company_list'
    with open(filename) as p_file:
        for line in p_file:
            total += 1

            if total <= startline:
                continue

            temp = line.strip().strip('\n').strip('\r')
            if temp == '':
                continue

            temp = temp.replace('#', '')
            temp = temp.replace('@', '')
            temp = temp.replace('!', '')
            temp = temp.replace('$', '')
            temp = temp.replace('%', '')
            temp = temp.replace('&', '')
            temp = temp.replace('*', '')
            temp = temp.replace(',', '')
            temp = temp.replace('，', '')
            temp = temp.replace(';', '')
            temp = temp.replace('?', '')
            temp = temp.replace('~', '')
            temp = temp.replace('\\', '')

            if len(temp) <= 9:
                continue

            count += 1

            result_list.append({
                '_id': temp,
                '_in_time': util.get_now_time(),
            })
            if len(result_list) >= 500:
                source_db.insert_batch_data(source_table, result_list)
                del result_list[:]

            if total % 1000 == 0:
                log.info("当前导入进度: total = {} count = {}".format(total, count))

                # log.info("length = {} company = {}".format(len(temp), temp))
    if len(result_list) > 0:
        source_db.insert_batch_data(source_table, result_list)
        del result_list[:]


if __name__ == '__main__':
    main()
